# Updated by Wei on 2013/09/01 at school 

from __future__ import division
from operator import itemgetter, attrgetter

import math
import matplotlib
import os
import pylab
import random
import sys
import time

# modularized by Wei on 2013/09/01 at school
def plot_sth_weeks_ago():
    # working example in both local computer and remote pangolin server
    # step1: select the random samples
    randomlySelectedLineIndexList = []
    randomlySelectedLineIndexDict = {}
    
    # in debug
    # totalNumOfRandomlySampledSamples = 10
    # in production
    totalNumOfRandomlySampledSamples = 1000000
    totalNumOfDocuments = 25205179
    # assume that the lineIndex starts from 0 to 25205179 in the total of 25,205,179 (25M)
    while len(randomlySelectedLineIndexDict) != totalNumOfRandomlySampledSamples:
        # Return a random integer N such that a <= N <= b
        lineIndex = random.randint(0, totalNumOfDocuments-1)
        if lineIndex not in randomlySelectedLineIndexDict:
            randomlySelectedLineIndexDict[lineIndex] = 1
            randomlySelectedLineIndexList.append(lineIndex)
    print "len(randomlySelectedLineIndexDict):",len(randomlySelectedLineIndexDict)
    print "len(randomlySelectedLineIndexList):",len(randomlySelectedLineIndexList)
    randomlySelectedLineIndexList.sort(cmp=None, key=None, reverse=False)
    print "randomlySelectedLineIndexList[-2]:",randomlySelectedLineIndexList[-2]
    print "randomlySelectedLineIndexList[-1]:",randomlySelectedLineIndexList[-1]
    time.sleep(10) # delays for 10 seconds
    # print randomlySelectedLineIndexDict
    
    NUM_OF_LINES_NEEDED = 26000000
    x = []
    y = []
    
    inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/trecID_docID_numOfPostingsRecorded_DocSizeInWords_MappingTableForGov2Dataset_with_Xdoc_values_added_sorted_by_XdocValueUsingGoodTurningDividedByNumOfPostingsForEachDocument"
    inputFileHandler = open(inputFileName,"r")
    
    
    currentLine = inputFileHandler.readline()
    currentLineIndex = 0
    
    while currentLine and currentLineIndex < NUM_OF_LINES_NEEDED:
        currentLineElements = currentLine.strip().split(" ")
        currentNumOfPostingsRecorded = int(currentLineElements[2])
        currentXdocValue = float(currentLineElements[7])
        
        if currentLineIndex in randomlySelectedLineIndexDict:
            x.append(currentNumOfPostingsRecorded)
            y.append(currentXdocValue)
            print "len(x):",len(x)
            print "currentLineIndex:",currentLineIndex
            print
            
        
        currentLine = inputFileHandler.readline() 
        currentLineIndex += 1
    
    matplotlib.pyplot.scatter(x,y)
    matplotlib.pyplot.show()
    inputFileHandler.close()

def plot_partialBM25Score_probability_score():
    inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/partialBM25ScoreBucketingPrelimanryResultsV3_SMOOTHED_AND_ready_for_REGRESSION.csv"
    inputFileHandler = open(inputFileName,"r")
    # skip the headline first
    inputFileHandler.readline()
    
    x = []
    y = []
    
    for index,line in enumerate( inputFileHandler.readlines() ):
        lineElements = line.strip().split("\t")
        
        currentClassPartialBM25AVERAGEBound_X_Axis = float( lineElements[2] )
        currentClassProbability_Y_Axis = float( lineElements[5] )
        print index,lineElements[2],lineElements[5]
        
        x.append(currentClassPartialBM25AVERAGEBound_X_Axis)
        y.append(currentClassProbability_Y_Axis)

    matplotlib.pyplot.scatter(x,y)
    matplotlib.pyplot.show()
       
    inputFileHandler.close()

def plot_2D_impact_score_20131203Night():
    inputFileName = "/data3/obukai/the_new_trip_of_feature_generation/gov2ClearYourMindAndDoItAgain/impactWithLengthAnalyze20131130Night/probabiliesBasedOnNumeratorANDDenominator"
    inputFileHandler = open(inputFileName,"r")
    # skip the 3 head lines
    inputFileHandler.readline()
    inputFileHandler.readline()
    inputFileHandler.readline()
    
    pointCounts = 0
    x = []
    y = []
    for line in inputFileHandler.readlines():
        if line.strip().startswith("listLengthClassLabel:"):
            print line.strip()
            print "len(x):",len(x)
            print "len(y):",len(y)
            print "test in:"
            matplotlib.pyplot.scatter(x,y)
            matplotlib.pyplot.show()
            print "test out."
            print
            x = []
            y = []
        elif line.strip().startswith("--->:"):
            currentProbability = float( line.strip().split(" ")[5] )
            pointCounts += 1
            x.append(pointCounts)
            y.append(currentProbability)
    inputFileHandler.close()

def plot_test():
    print "test in:"
    x = []
    y = []
    x.append(1)
    y.append(1)
    
    x.append(2)
    y.append(2)
    matplotlib.pyplot.scatter(x,y)
    matplotlib.pyplot.show()    
    print "test out."
    
    print "test in:"
    x = []
    y = []
    x.append(3)
    y.append(3)
    
    x.append(4)
    y.append(4)    
    matplotlib.pyplot.scatter(x,y)
    matplotlib.pyplot.show()
    print "test out."
    
print "Program begins..."

plot_2D_impact_score_20131203Night()
# plot_test()

print "Program ends."
